from selenium import webdriver
import time
import re
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.by import By
import pymysql
import hashlib
import datetime

'''
拿到手机的详情页链接，存到数据库，
供xly调用
'''


class GLY(object):
    def __init__(self):
        self.url = 'https://www.jd.com/'
        # webdriver配置项，无头浏览器，降低损耗
        opt = webdriver.ChromeOptions()
        # opt.add_argument('--headless')
        opt.add_argument('--no-sandbox')
        opt.add_argument('--disable-gpu')
        opt.add_argument('--disable-dev-shm-usage')
        # 降配置添加到driver中
        self.driver = webdriver.Chrome(chrome_options=opt)

    def get_phone_urls(self, key):
        self.driver.get(self.url)
        # 等待输入框元素加载
        search_key = WebDriverWait(self.driver, 10).until(
            EC.presence_of_element_located((By.ID, 'key'))
        )
        # 输入框输入关键字
        search_key.send_keys(key)
        search_button = WebDriverWait(self.driver, 10).until(
            EC.presence_of_element_located((By.CLASS_NAME, 'button'))
        )
        # 模拟点击搜索按钮
        search_button.click()
        # 暂停5s（时间根据网速来定），等待网页加载（可能有更好的方法，但是目前不清楚）
        time.sleep(5)
        # 使用js动作模拟滚轮下滑，加载当前页全部
        for y in range(10):
            js = 'window.scrollBy(0,800)'
            self.driver.execute_script(js)
            time.sleep(0.5)
        # 暂停时间太短，源码加载不出来，所以暂定5s
        time.sleep(5)
        html = self.driver.page_source

        self.driver.quit()
        return html

    def parse_data(self, html):
        # 匹配页面中展示的所有手机id号，并格式化成链接
        phone_ids = re.findall('<li class="gl-item" data-sku="(\d+)"', html)
        data = {}
        for phone_id in phone_ids:
            link = 'https://item.jd.com/{}.html'.format(phone_id)
            hkey = hashlib.md5(link.encode(encoding='utf-8')).hexdigest()
            lasttime = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')
            data['link'] = link
            # 链接加密
            data['hkey'] = hkey
            # 抓取时间
            data['lasttime'] = lasttime
            # 标志是否为新链接，新链接为0，使用过为1
            data['tag'] = '0'
            self.save_data(data)
            print(phone_id)

    def save_data(self, data):
        host = '127.0.0.1'
        db = 'app_mark'
        user = 'root'
        passwd = '123456'
        charset = 'utf8mb4'
        con = pymysql.connect(host=host, db=db, user=user, passwd=passwd, charset=charset)
        cur = con.cursor()
        data_list = [data['link'], data['lasttime'], data['hkey'], data['tag']]
        sql = 'insert into gly(link, lasttime, hkey, tag) values (%s, %s, %s, %s)'
        try:
            cur.execute(sql, data_list)
        except Exception as e:
            con.rollback()
            print('error~', e)
        else:
            con.commit()
        cur.close()
        con.close()


if __name__ == '__main__':
    gly = GLY()
    key = '手机'
    html = gly.get_phone_urls(key)
    gly.parse_data(html)

